<?php
include_once('../lib/config.php');
include_once('../lib/Db.class.php');
include_once('../lib/Model.class.php');
include_once('../lib/Snoopy.class.php');
include_once('../lib/parseHtml.class.php');
include_once('../lib/function.php');
include_once('/usr/local/xunsearch/sdk/php/lib/XS.php');
/**/
// from search_chapter2.php
$redis = new Redis();  
$ret = $redis->connect("localhost","6379");  //php客户端设置的ip及端口
if(!$ret) die('redis connction faild');
$redis->auth(REDIS_PASS);
$redis->select(2);

$GLOBALS['config']['master']['db_database'] = 'gouyg';
$GLOBALS['config']['slave']['db_database']  = 'gouyg';

$basedir = dirname(__FILE__);
$model = new Db;
$parseHtml = new parseHtml;
$snoopy = new Snoopy;
$snoopy->_redirectaddr = true;
$limit = 100;
$use_proxy = true;
while ($limit--) {
    echo2('limit:'.$limit.', use_proxy:'.intval($use_proxy));
    try {
        $q_string = $redis->lPop('BOOK_CONTENT_QUEUE2');
        if(!$q_string) {
            echo2('BOOK_CONTENT_QUEUE2 empty, limit:'.$limit);
            sleep(2);
            continue;
        }

        $data = json_decode($q_string, true);
        list($md5id, $url, $chapter_id) = $data;

        $cachefile_old = $basedir .'/read_all/'.$md5id.'/'.urlencode($url);
        $localfile = '/read_all/'.$md5id.'/'.md5($url);
        $cachefile = $basedir . $localfile;
        if(!file_exists($basedir .'/read_all/'.$md5id)) {
            mkdir($basedir .'/read_all/'.$md5id);
        }

        if(file_exists($cachefile_old)) {
            rename($cachefile_old, $cachefile);
        }

        if($chapter_id) {
            $old_chapter = $model->find('book_chapter', "id='{$chapter_id}'");
            if($old_chapter && $old_chapter['remote_file'] == 1) {
                echo2($cachefile.' remote exists');
                continue;
            }
        }

        if(file_exists($cachefile)) {
            echo2($cachefile.' exists');
            continue;
        }

        // 获取url采集锁
        $locked_remove = true;
        $locked_key = 'locked:'.urlencode($url);
        $locked = $redis->setNx($locked_key, time());
        if($locked) {
            $redis->expire($locked_key, 300);
        } else {
            $locked_remove = false;
            throw new Exception('locked url:'.$url);
        }
        $snoopy->curl($url, $use_proxy);

        if(in_array($snoopy->status, [0, 401])) {
            $use_proxy = !$use_proxy;
        }

        $parseResult = $parseHtml->getContent($url, $snoopy->results, $snoopy);
        if(!$parseResult) {
            throw new Exception('该url地址没有相应解析规则');
        }

        list($title, $body) = $parseResult;
        if(empty($title) || empty($body)) {
            if($snoopy->status == 200 && !empty($title)) {
                $q_string = null;
                if($chapter_id) {
                    $model->save('book_chapter', "id='{$chapter_id}'", ['local_empty'=>1]);
                }
            }

            throw new Exception('fetch error，status:'.$snoopy->status.'，url:'.$url);
        }

        $r = file_put_contents($cachefile, 'jcall('.json_encode(['title'=>$title, 'body'=>$body]).')');
        if(!$r) {
            throw new Exception('cachefile failed');
        }

        // 更新chapter信息
        if($chapter_id) {
            $model->save('book_chapter', "id='{$chapter_id}'", ['local_file'=>$localfile, 'local_empty'=>0]);
        }

        $local_count = $redis->incr('chapters:'.$md5id);
        
        //释放锁
        $redis->delete($locked_key);
        echo2('chapter_id:'. $chapter_id . ', ' . $title.', local_count:'.$local_count.', cachefile:'.$cachefile);

    } catch (Exception $e) {
        echo2($e->getMessage());

        if($locked_remove) {
            $redis->delete($locked_key);
        }
        
        if($q_string) {
            $redis->rPush('BOOK_CONTENT_QUEUE2', $q_string);
        }

        sleep(1);
    }
}

sleep(2);
